第三篇 K8S集群部署之ETCD集群部署

本文阅读量: 2019-11-04

1 自签ETCD SSL证书

1.1 安装cfssl

curl -L https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 -o /usr/local/bin/cfssl
curl -L https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 -o /usr/local/bin/cfssljson
curl -L https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 -o /usr/local/bin/cfssl-certinfo
chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssljson /usr/local/bin/cfssl-certinfo

1.2 生成自签证书

mkdir -p /root/k8s/etcd-cert
cd /root/k8s/etcd-cert
cat > ca-config.json <<EOF
{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "www": {
         "expiry": "87600h",
         "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ]
      }
    }
  }
}
EOF

cat > ca-csr.json <<EOF
{
    "CN": "etcd CA",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "Beijing",
            "ST": "Beijing"
        }
    ]
}
EOF

cfssl gencert -initca ca-csr.json | cfssljson -bare ca -

#-----------------------

cat > server-csr.json <<EOF
{
    "CN": "etcd",
    "hosts": [
    "192.168.0.201",
    "192.168.0.211",
    "192.168.0.212"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "BeiJing",
            "ST": "BeiJing"
        }
    ]
}
EOF

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=www server-csr.json | cfssljson -bare server

2 创建etcd集群脚本并执行脚本

2.1 创建etcd.sh 脚本

mkdir -p /root/shell-script
cd /root/shell-script
vim etcd.sh
#!/bin/bash
# example: ./etcd.sh etcd01 192.168.1.10 etcd02=https://192.168.1.11:2380,etcd03=https://192.168.1.12:2380

ETCD_NAME=$1
ETCD_IP=$2
ETCD_CLUSTER=$3

WORK_DIR=/opt/etcd

cat <<EOF >$WORK_DIR/cfg/etcd
#[Member]
ETCD_NAME="${ETCD_NAME}"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://${ETCD_IP}:2380"
ETCD_LISTEN_CLIENT_URLS="https://${ETCD_IP}:2379"

#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://${ETCD_IP}:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://${ETCD_IP}:2379"
ETCD_INITIAL_CLUSTER="${ETCD_NAME}=https://${ETCD_IP}:2380,${ETCD_CLUSTER}"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
ETCD_ENABLE_V2="true"
EOF

cat <<EOF >/usr/lib/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target

[Service]
Type=notify
EnvironmentFile=${WORK_DIR}/cfg/etcd
ExecStart=${WORK_DIR}/bin/etcd \
--cert-file=${WORK_DIR}/ssl/server.pem \
--key-file=${WORK_DIR}/ssl/server-key.pem \
--peer-cert-file=${WORK_DIR}/ssl/server.pem \
--peer-key-file=${WORK_DIR}/ssl/server-key.pem \
--trusted-ca-file=${WORK_DIR}/ssl/ca.pem \
--peer-trusted-ca-file=${WORK_DIR}/ssl/ca.pem
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl enable etcd
systemctl restart etcd

2.2 启动etcd集群

cp /root/k8s/etcd-cert/ca*pem /root/k8s/etcd-cert/server*pem /opt/etcd/ssl
mkdir /opt/etcd/{bin,cfg,ssl} -p
mkdir /root/software
cd /root/software
wget https://github.com/etcd-io/etcd/releases/download/v3.4.3/etcd-v3.4.3-linux-amd64.tar.gz
tar -zxvf etcd-v3.4.3-linux-amd64.tar.gz
mv etcd-v3.4.3-linux-amd64/{etcd,etcdctl} /opt/etcd/bin/
cd /root/shell-script
chmod u+x etcd.sh
./etcd.sh etcd01 192.168.0.201 etcd02=https://192.168.0.211:2380,etcd03=https://192.168.0.212:2380

3 启动另外两个etcd节点

# 先复制证书和配置等信息
scp -r /opt/etcd/ root@192.168.0.211:/opt/
scp -r /opt/etcd/ root@192.168.0.212:/opt/
scp /root/shell-script/etcd.sh root@192.168.0.211:/root/etcd.sh
scp /root/shell-script/etcd.sh root@192.168.0.212:/root/etcd.sh
# 在192.168.0.211
./etcd.sh etcd02 192.168.0.211 etcd01=https://192.168.0.201:2380,etcd03=https://192.168.0.212:2380
# 在192.168.0.212
./etcd.sh etcd03 192.168.0.212 etcd01=https://192.168.0.201:2380,etcd02=https://192.168.0.211:2380

4 检查集群启动是否成功 etcdctl v3

$ /opt/etcd/bin/etcdctl --endpoints=https://192.168.0.201:2379 --key=/opt/etcd/ssl/server-key.pem --cert=/opt/etcd/ssl/server.pem --cacert=/opt/etcd/ssl/ca.pem member list
d1e97ac3dce77936, started, etcd03, https://192.168.0.212:2380, https://192.168.0.212:2379, false
d289ffc8e8b8fb28, started, etcd01, https://192.168.0.201:2380, https://192.168.0.201:2379, false
f151a08f96159b75, started, etcd02, https://192.168.0.211:2380, https://192.168.0.211:2379, false

4 检查集群启动是否成功 etcdctl v2

$ export ETCDCTL_API=2
$ /opt/etcd/bin/etcdctl \
--ca-file=/opt/etcd/ssl/ca.pem \
--cert-file=/opt/etcd/ssl/server.pem \
--key-file=/opt/etcd/ssl/server-key.pem \
--endpoints="https://192.168.0.201:2379,https://192.168.0.211:2379,https://192.168.0.212:2379" \
cluster-health
member d1e97ac3dce77936 is healthy: got healthy result from https://192.168.0.212:2379
member d289ffc8e8b8fb28 is healthy: got healthy result from https://192.168.0.201:2379
member f151a08f96159b75 is healthy: got healthy result from https://192.168.0.211:2379